package com.tedu;

import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

/**
 * 爬虫页面解析
 */
public class PageProcessoriMPL implements PageProcessor {

    //site对象
    Site site = Site.me().setRetryTimes(5).setTimeOut(4000);
    //定义正则表达式
    private String reg = "http://www.dangjian.com/[\\w/.stml]+";

    public void process(Page page) {

        Html html = page.getHtml();
//        System.out.println("标题："+html.getDocument().title());

        //全部的超链接,清洗后数据，全部都是党建网的数据 //数据清洗
        Selectable regex = html.links().regex(reg);

        //把新的uml添加到webmagic框架，进行爬取
        page.addTargetRequests(regex.all());

        //解析页面内容
        Selectable url = page.getUrl();
        List<String> all = url.all();

//        System.out.println("url:"+all.get(0));
        //.shtml  解析页面内容 党建信息类型+内容  标题，来源，时间，作者，内容
        //判断当前爬取的页面是否是党建信息页面
        if(all.size()==0){
            return;
        }
        String[] split = all.get(0).split(".");

        boolean contains = all.get(0).contains(".shtml");
        if(contains){
            System.out.println("当前页面是党建信息页面");
        }

        //页面的解析
        try{
            Document document = html.getDocument();
            //获取标题类 title_tex
            String title_tex = document.getElementById("title_tex").text();
            //发表时间
            String time_tex = document.getElementById("time_tex").text();
            //来源
            String time_ly = document.getElementById("time_ly").text();
            //内容
            Elements elementsByClass = document.getElementsByClass("lx-box1");
            //作者
            String bj = document.getElementsByClass("bj").text();

            //类型
            String currChnlCls = document.getElementsByClass("CurrChnlCls").text();

//            System.out.println("title_tex="+title_tex+";elementsByClass="+elementsByClass);

            //把数据发送到爬虫的数据持久化组件中
            page.putField("title_tex",title_tex);
            page.putField("time_tex",time_tex);
            page.putField("time_ly",time_ly);
            page.putField("elementsByClass",elementsByClass);

            page.putField("bj",bj);
            page.putField("currChnlCls",currChnlCls);//类型


        }catch (Exception e){
//            e.printStackTrace();
        }


    }

    public Site getSite() {
        return site;
    }
}
